<?php
// +----------------------------------------------------------------------
// | ThinkCMF [ WE CAN DO IT MORE SIMPLE ]
// +----------------------------------------------------------------------
// | Copyright (c) 2013-2018 http://www.thinkcmf.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: guosheng1987@126.com 
// +----------------------------------------------------------------------
namespace plugins\crawler\service; 

use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Pool;
use GuzzleHttp\Promise;
use GuzzleHttp\Cookie\CookieJar;

use phpQuery;
//use QL\QueryList;

use plugins\crawler\model\CrawlerSiteModel;
use plugins\crawler\model\CrawlerUrlModel;
use plugins\crawler\model\CrawlerRuleModel;
use plugins\crawler\model\CrawlerCollectionModel;

use app\portal\model\PortalPostModel;
use app\portal\model\PortalCategoryModel;

use think\Db;

class CrawlerDoubanService extends CrawlerService
{   
    //GuzzleHttp Client 
    public $client;
    //http headers
    public $headers;
    // current crawler site object
    public $site;
    // current crawler url object
    public $url;

    public $collections;

    public $serverConfig = [
        'data' => [
            'client' => [
                // Base URI is used with relative requests
                'base_uri' => 'https://movie.douban.com',
                // default options.
                'timeout'  => 30.0,
                'verify'   => false,
                'delay'    => 3000,
            ],
            'headers' => [
                'User-Agent'      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
                'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
                'Accept-Encoding' => 'gzip, deflate',
                'Accept-Language' => 'zh-CN,zh;q=0.9',
                'Cache-Control'   => 'max-age=0',
                'Connection'      => 'keep-alive',
                'Host'            => 'movie.douban.com',
                'Cookie'          => '',
                'Referer'         => 'https://movie.douban.com/tv/',
                'X-Requested-With'=> 'XMLHttpRequest'

            ]
        ],
        'image' => [
            'client' => [
                // Base URI is used with relative requests
                'base_uri' => 'https://img3.doubanio.com/',
                // default options.
                'timeout'  => 8.0,
                'verify'   => false,
                'delay'    => 3000,
            ],
            'headers' => [
                'User-Agent'      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
                'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
                'Accept-Encoding' => 'gzip, deflate',
                'Accept-Language' => 'zh-CN,zh;q=0.9',
                'Cache-Control'   => 'max-age=0',
                'Connection'      => 'keep-alive',
                'Cookie'          => 'n7qUgTZmP4M',
            ]
        ],
    ];
    /**
     * 构造函数
     *
     */
    public function __construct() {}   

    
    /**
     * 设置请求头信息
     * @access public
     * @return void
     */
    public function setCookie($headers)
    {
        $time = time();
        $cookie = [
            'bid=-zwhAvkawQY',
            'll=118254',
            '_vwo_uuid_v2=30CEDC984B4EF815AA57879CC68F8F2C|390e1cf5b17fd4d06826d145c870b43e',
            '__yadk_uid=yTm8kLIcKUgCsqVp7xpzDfmHD3mmmNNS',
            'ap=1', 
            'ps=y',
            '_pk_id.100001.4cf6=2d64c626a6321cda.'.$time.'.20.'.($time-312).'.'.$time.'.',
            '_pk_ref.100001.4cf6='. urlencode('["","",'.$time.',"https://www.douban.com/"]'),
            '_pk_ses.100001.4cf6=*',
            '__utma=30149280.2069107026.1222325434.'.($time-312).'.'.$time.'.29',
            '__utmb=30149280.0.10.'.$time, 
            '__utmc=30149280',
            '__utmz=30149280.'.$time.'.23.16.utmcsr=baidu|utmccn=(organic)|utmcmd=organic',
            '__utma=223695111.1817819947.'.$time.'.'.$time.'.'.$time.'21', 
            '__utmb=223695111.0.10.'.$time,
            '__utmc=223695111', 
            '__utmz=223695111.'.$time.'.20.18.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/',
        ];
        
        $headers['Cookie'] = implode('; ', $cookie);

        return $headers;
    }

    /**
     * get collection path and Thumb img from list page
     * @access public
     * @param  String $contents
     * @return Array $collection
     */
    public function parseCollectionInfoByContents($contents)
    {
        $collections = [];

        if($contents)
        {
            $result = json_decode($contents);
            
            foreach($result->subjects as $subject)
            {
                $item['paths']            = $subject->url;
                $item['title']            = $subject->title;
                $item['playable']         = $subject->playable;
                $item['collection_id']    = $subject->id;

                $collections[] = $item;
            }
        }

        return $collections;
    }

    /**
     * parse actor table from movie page html
     * @access public
     * @param  String $contents
     * @return String $filmmaker
     */

    public function parseCollectionFilmmakerByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $filmmaker = pq('div.article div.subject.clearfix div#info')->html();

            phpQuery::$documents = null;
        }

        return $filmmaker;
    }

    /**
     * parse Synopsis from movie page html
     * @access public
     * @param  String $contents
     * @return String $synopsis
     */

    public function parseCollectionSynopsisByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $filmmaker = pq('div#link-report')->text();

            phpQuery::$documents = null;
        }

        return $filmmaker;
    }

    /**
     * parse Thumb image from movie page html
     * @access public
     * @param  String $contents
     * @return String $imagePath
     */
    public function parseCollectionThumbImageByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $imagePath = pq('div.article div.subject.clearfix div#mainpic')->find('img')->attr('src');

            phpQuery::$documents = null;

        }

        return $imagePath;
    }

    /**
     * parse comments from article page html
     * @access public
     * @param  String $contents
     * @return String $comments
     */
    public function parseCollectionCommentsByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents, 'utf-8');
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $comments = '';

            foreach(pq('div#hot-comments div.comment-item') as $div)
            {
                $comment['user']     = pq($div)->find('div.comment span.comment-info a')->text();
                $comment['time']     = pq($div)->find('div.comment span.comment-info span.comment-time')->attr('title');
                $comment['votes']    = pq($div)->find('div.comment span.comment-vote span')->text();
                $comment['content']  = pq($div)->find('div.comment p')->text();
                
                $comments .= '<div class="row" style="margin:5px 5px 5px 0px;">
                <p>
                <span class="pull-left"><a>'.$comment['user'].'</a>&nbsp;<span class="comment-time">'. $comment['time'] .'</span></span>
                <a class= "pull-right">
                    <i class="fa fa-thumbs-up">
                    <span class="count">'.$comment['votes'].'</span>
                    </i>
                </a>
                </p></div>';
                $comments .= "<p>".$comment['content'] ."<p>";
            }

            phpQuery::$documents = null;
        }

        return $comments;
    }


    /**
     * request list url and aget 
     * @access public
     * @return string
     */
    public function reuqestThumbImage($imagePath)
    {
        $client = new Client($this->serverConfig['image']['client']);

        $response = $client->request('GET', $imagePath, [
            'allow_redirects' => false,
            //'headers'         => $this->headers,
        ]);
        
        $localFile = $this->saveLocalFile($response->getBody()->getContents(), $imagePath);
        
        return $localFile['url'];
    }
    
    /**
     * generate rand accounts
     * @access public
     * @return Array $accounts
     */
    public function getRandAccounts()
    {
        $maxid = Db::name('accounts')->max('id');
        $times = rand(3,8);
        $ids = [];
        
        while(true)
        {
            $id = rand(1, $maxid);
            if( !in_array($id, $ids) ) {
                $ids[] = $id;
            }
            if(count($ids) == $times) break;
        }
        
        $accounts = Db::name('accounts')
            ->where(['id' => ['in', $ids]])
            ->select();

        return $accounts;
    }

   
    /**
     * generate portal data
     * @access private
     * @param  String $categoryName
     * @param  Array $collection 
     * @return Array data
     */
    private function makePortalData($categoryName, $collection)
    {        
        $data = [];
        $content = '';

        $data['title']     = date('Y').$collection['title'];
        $data['thumbnail'] = $collection['thumbs'];
        $data['excerpt']   = mb_substr($collection['synopsis'], 0, 150);
        $sptpl =
<<<EOF
<h2 style="box-sizing: border-box; font-family: 'Microsoft YaHei', 'Source Sans Pro', Calibri, Candara, Arial, sans-serif; font-weight: 400; line-height: 1.3em; color: rgb(51, 51, 51); margin: 14px 0px; font-size: 16px; white-space: normal; background-color: rgb(255, 255, 255); padding: 2px 0px 2px 10px; border-left: 4px solid rgb(64, 153, 208);"><span style="box-sizing: border-box; font-family: suxingme, 'Open Sans', Arial, 'Hiragino Sans GB', 'Microsoft YaHei', STHeiti, 'WenQuanYi Micro Hei', SimSun, sans-serif;">%s</span></h2>
EOF;
        
        $content = sprintf($sptpl, $collection['title']);
        $content .= '<p>'. $collection['filmmaker'] .'</p>';
        $content .= '<p><br/></p>';
        $content .= sprintf($sptpl, '剧情介绍');
        $content .= '<p>'. $collection['synopsis'] .'</p>';
        $content .= '<p><br/></p>';
        $content .= sprintf($sptpl, '网友热评');
        $content .= '<p>'. $collection['comments'] .'</p>';
        $content .= '<p><br/></p>';

        $data['content'] = $content;

        return $data;
    } 

    
    /**
     * save collection to db
     * @access public
     * @param  Int $index 
     * @return void
     */
    public function saveCrawlerData($index)
    {
        //先删除关联表，防止重复
        $this->deletePortal($this->collections[$index]['collection_id']);
        
        $categoryId = $this->url->crlrule->category_id;
        
        $categoryName = $this->getPortalCategoryNameById($categoryId);

        $portal = $this->makePortalData($categoryName, $this->collections[$index]);
        
        $postId = $this->addPortalAndTag($categoryId, $portal['title'], $portal['excerpt'] ,$portal['content'], $portal['thumbnail']);
        
        $this->addCrawlerCollection($postId, $this->collections[$index]);
    }
    
    /**
     * request list url and aget 
     * @access public
     * @return string
     */
    public function reuqestListPage()
    {
        $client = new Client($this->serverConfig['data']['client']);
        
        $headers = $this->serverConfig['data']['headers'];

        $headers = $this->setCookie($headers);

        $response = $client->request('GET', $this->url->listurl, [
            'allow_redirects' => false,
            'headers'         => $headers,
        ]);
        
        $contents = $response->getBody()->getContents();

        return $contents;
    }

    /**
     * request Qrcode image url in article page
     * @access public
     * @return void
     */
    public function requestCollectionPages()
    {
        $client = new Client($this->serverConfig['data']['client']);

        $headers = $this->serverConfig['data']['headers'];

        $headers = $this->setCookie($headers);

        $requests = function ($headers) {
            foreach ($this->collections as $i => $collection) {
                yield new Request('GET', $collection['paths'], $headers);                                
            }
        };

        $pool = new Pool($client, $requests($headers), [
            'concurrency' => 3,
            'fulfilled'   => function ($response, $index) {
                
                $contents = $response->getBody()->getContents();

                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' start!'); 
                //yingren xinxi 
                $filmmaker = $this->parseCollectionFilmmakerByContents($contents);                               
                //juqing jianjie
                $synopsis = $this->parseCollectionSynopsisByContents($contents);
                
                $comments = $this->parseCollectionCommentsByContents($contents);
                
                $imagePath = $this->parseCollectionThumbImageByContents($contents);
                
                $thumbs = $this->reuqestThumbImage($imagePath);

                $this->collections[$index]['filmmaker']       = $filmmaker;
                $this->collections[$index]['synopsis']        = $synopsis;
                $this->collections[$index]['comments']        = $comments;
                $this->collections[$index]['thumbs']          = $thumbs;
                $this->collections[$index]['content']         = $filmmaker.$synopsis.$comments;

                $this->saveCrawlerData($index);
            
                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' collect data success!');    
                 
            },
            'rejected'   => function ($reason, $index) {
                
                //var_dump($reason);

                // this is delivered each failed request
                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' request fail!');

            },
        ]);

        // Initiate the transfers and create a promise
        $promise = $pool->promise();

        // Force the pool of requests to complete.
        $promise->wait();
        
    }
    
    /**
     * get Collection ids not in table
     * @access public
     * @return void
     */
    public function getNoSavedCollectionIds()
    {
        $currentIds = $savedIds = [];
        foreach($this->collections as $i => $collection)
        {
            $currentIds[$i] = $collection['collection_id'];
        }

        $savedIds = Db::name('crawler_collection')
            ->where(['crawler_site_id' => $this->site->id])
            ->where(['collection_id' => ['in', $currentIds]])
            ->column('collection_id');

        foreach($currentIds as $i => $collection_id)
        {
            if( in_array($collection_id, $savedIds) )
            {
                unset($this->collections[$i]);
            }
        }
        
        $this->collections = array_values($this->collections);

    }

    /**
     * start crawler job
     * 
     * @return array
     */
    public function start()
    {
        $this->site = CrawlerSiteModel::find(4);

        $urls = CrawlerUrlModel::where('crawler_site_id', $this->site->id)->select();
        
        foreach($urls as $url)
        {   
            $this->url = $url;
           
            $contents = $this->reuqestListPage();

            $this->output->writeln("list url: ". $this->url->listurl.' request success!');
            
            //var_dump($contents);die;

            $this->collections = $this->parseCollectionInfoByContents($contents);
    
            /*
            this is for test special page 
            $collections = $this->collections;
            $tmpCol = [];
            foreach($collections as $i => $collection)
            {
                $matchtimes = preg_match('/\d+/', $collection['paths'], $matches);

                if($matches[0] == '460951961345051205')
                {
                    $tmpCol[] = $collection;
                }
            }

            $this->collections = $tmpCol;
            */
            
            $this->getNoSavedCollectionIds();
            
            if(count($this->collections))
            {
                $this->collections = array_slice($this->collections, 0, 2);
                
                $this->requestCollectionPages();
                
                $this->output->writeln("list url ".$this->url->listurl.' collected success!');
            }
            else
            {
                $this->output->writeln("list url ".$this->url->listurl.' already done!');
            }
        }
        $this->output->writeln("Command executed success!");
        
    }
    
}